#ifdef __aarch64__

.text
.align 5
.global IndirectGemmFp16_16x8
#ifndef __APPLE__
.type IndirectGemmFp16_16x8, %function
#endif

// void IndirectGemmFp16_16x8(float16_t *output, float16_t *input, float16_t *weight, float16_t *bias,
//                                                                 size_t step, size_t ic4, size_t oc8, size_t offset, size_t relu, size_t relu6);
// x0: output, x1: input, x2: weight, x3: bias, x4: step, x5: ic4, x6: oc8, x7: offset, x8:relu, x9: relu6
// compute 8 channel for 16 outputs
IndirectGemmFp16_16x8:

    .macro INIT_BIAS
        ld1 {v16.8h}, [x3]
        mov v17.8h, v16.8h
        mov v18.8h, v16.8h
        mov v19.8h, v16.8h
        mov v20.8h, v16.8h
        mov v21.8h, v16.8h
        mov v22.8h, v16.8h
        mov v23.8h, v16.8h
        mov v24.8h, v16.8h
        mov v25.8h, v16.8h
        mov v26.8h, v16.8h
        mov v27.8h, v16.8h
        mov v28.8h, v16.8h
        mov v29.8h, v16.8h
        mov v30.8h, v16.8h
        mov v31.8h, v16.8h
    .endm

    // registers v8 ~ v15 must be preserved by a callee across subroutine calls, according to
    // https://github.com/ARM-software/abi-aa/blob/master/aapcs64/aapcs64.rst#simd-and-floating-point-registers
    // x19 ~ r29 should be also preserved
    // whereas our coding style do not permit such amount of parameters
    sub sp, sp, #64
    // performance between storing 4 registers at the same time and seperatly storing them on in-order cores
    // is not tested yet
    st1 {v8.4s, v9.4s, v10.4s, v11.4s}, [sp], #64

    ldr x8, [sp, #0]
    ldr x9, [sp, #8]

    // step is one for common convolution, where ic8 should multiply by kernel size 
    // step is (a+b-1) for F(a,b) in winograd
    mul x5, x4, x5
    mov x4, #1

    LoopStart:

        mov x10, x4
        mov x12, x1

        LoopKw:

                mov x11, x0
                INIT_BIAS
                // load input for output 1-8
                ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x12], #64
                // load weight
                ld1 {v8.8h, v9.8h}, [x2], #32
                //  first 2 steps for output 1 and 3
                fmla v16.8h, v8.8h, v0.h[0]
                fmla v18.8h, v8.8h, v1.h[0]
                fmla v16.8h, v9.8h, v0.h[1]
                fmla v18.8h, v9.8h, v1.h[1]
                // load weight
                ld1 {v10.8h, v11.8h}, [x2], #32
                //  first 2 steps for output 2 and 4
                fmla v17.8h, v8.8h, v0.h[4]
                fmla v19.8h, v8.8h, v1.h[4]
                fmla v17.8h, v9.8h, v0.h[5]
                fmla v19.8h, v9.8h, v1.h[5]
                // load input  for output 9-16
                // input cache should be refreshed after loading
                // ATTENTION: advance is prefered, but advancing too much may lead to invalid prefetching 
                ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x12], #64
                //  last 2 steps for output 1 and 3
                fmla v16.8h, v10.8h, v0.h[2]
                fmla v18.8h, v10.8h, v1.h[2]
                fmla v16.8h, v11.8h, v0.h[3]
                fmla v18.8h, v11.8h, v1.h[3]

                // check if ic4=1
                subs x13, x5, #1
                beq LoopIcEnd

            LoopIc:
                //  last 2 steps for output 2 and 4
                fmla v17.8h, v10.8h, v0.h[6]
                fmla v19.8h, v10.8h, v1.h[6]
                fmla v17.8h, v11.8h, v0.h[7]
                fmla v19.8h, v11.8h, v1.h[7]
                //  steps for output 5-8
                fmla v20.8h, v8.8h, v2.h[0]
                fmla v22.8h, v8.8h, v3.h[0]
                fmla v20.8h, v9.8h, v2.h[1]
                fmla v22.8h, v9.8h, v3.h[1]
                fmla v21.8h, v8.8h, v2.h[4]
                fmla v23.8h, v8.8h, v3.h[4]
                fmla v21.8h, v9.8h, v2.h[5]
                fmla v23.8h, v9.8h, v3.h[5]
                fmla v20.8h, v10.8h, v2.h[2]
                fmla v22.8h, v10.8h, v3.h[2]
                fmla v20.8h, v11.8h, v2.h[3]
                fmla v22.8h, v11.8h, v3.h[3]
                fmla v21.8h, v10.8h, v2.h[6]
                fmla v23.8h, v10.8h, v3.h[6]
                fmla v21.8h, v11.8h, v2.h[7]
                fmla v23.8h, v11.8h, v3.h[7]
                // load input for output 1-8
                ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x12], #64
                //  steps for output 9-12
                fmla v24.8h, v8.8h, v4.h[0]
                fmla v26.8h, v8.8h, v5.h[0]
                fmla v24.8h, v9.8h, v4.h[1]
                fmla v26.8h, v9.8h, v5.h[1]
                fmla v25.8h, v8.8h, v4.h[4]
                fmla v27.8h, v8.8h, v5.h[4]
                fmla v25.8h, v9.8h, v4.h[5]
                fmla v27.8h, v9.8h, v5.h[5]
                fmla v24.8h, v10.8h, v4.h[2]
                fmla v26.8h, v10.8h, v5.h[2]
                fmla v24.8h, v11.8h, v4.h[3]
                fmla v26.8h, v11.8h, v5.h[3]
                fmla v25.8h, v10.8h, v4.h[6]
                fmla v27.8h, v10.8h, v5.h[6]
                fmla v25.8h, v11.8h, v4.h[7]
                fmla v27.8h, v11.8h, v5.h[7]
                //  steps for output 13-16
                fmla v28.8h, v8.8h, v6.h[0]
                fmla v30.8h, v8.8h, v7.h[0]
                fmla v28.8h, v9.8h, v6.h[1]
                fmla v30.8h, v9.8h, v7.h[1]
                fmla v29.8h, v8.8h, v6.h[4]
                fmla v31.8h, v8.8h, v7.h[4]
                fmla v29.8h, v9.8h, v6.h[5]
                fmla v31.8h, v9.8h, v7.h[5]
                // load weight
                ld1 {v8.8h, v9.8h}, [x2], #32
                fmla v28.8h, v10.8h, v6.h[2]
                fmla v30.8h, v10.8h, v7.h[2]
                fmla v28.8h, v11.8h, v6.h[3]
                fmla v30.8h, v11.8h, v7.h[3]
                fmla v29.8h, v10.8h, v6.h[6]
                fmla v31.8h, v10.8h, v7.h[6]
                fmla v29.8h, v11.8h, v6.h[7]
                fmla v31.8h, v11.8h, v7.h[7]
                // load weight
                ld1 {v10.8h, v11.8h}, [x2], #32
                // first 2 steps for output 1-4
                fmla v16.8h, v8.8h, v0.h[0]
                fmla v18.8h, v8.8h, v1.h[0]
                fmla v16.8h, v9.8h, v0.h[1]
                fmla v18.8h, v9.8h, v1.h[1]
                fmla v17.8h, v8.8h, v0.h[4]
                fmla v19.8h, v8.8h, v1.h[4]
                fmla v17.8h, v9.8h, v0.h[5]
                fmla v19.8h, v9.8h, v1.h[5]
                // load input  for output 9-16
                ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x12], #64
                //  last 2 steps for output 1 and 3
                fmla v16.8h, v10.8h, v0.h[2]
                fmla v18.8h, v10.8h, v1.h[2]
                fmla v16.8h, v11.8h, v0.h[3]
                fmla v18.8h, v11.8h, v1.h[3]

                subs x13, x13, #1
                bne LoopIc

            LoopIcEnd:
                fmla v17.8h, v10.8h, v0.h[6]
                fmla v19.8h, v10.8h, v1.h[6]
                fmla v17.8h, v11.8h, v0.h[7]
                fmla v19.8h, v11.8h, v1.h[7]
                //  steps for output 5-8
                fmla v20.8h, v8.8h, v2.h[0]
                fmla v22.8h, v8.8h, v3.h[0]
                fmla v20.8h, v9.8h, v2.h[1]
                fmla v22.8h, v9.8h, v3.h[1]
                fmla v21.8h, v8.8h, v2.h[4]
                fmla v23.8h, v8.8h, v3.h[4]
                fmla v21.8h, v9.8h, v2.h[5]
                fmla v23.8h, v9.8h, v3.h[5]
                fmla v20.8h, v10.8h, v2.h[2]
                fmla v22.8h, v10.8h, v3.h[2]
                fmla v20.8h, v11.8h, v2.h[3]
                fmla v22.8h, v11.8h, v3.h[3]
                fmla v21.8h, v10.8h, v2.h[6]
                fmla v23.8h, v10.8h, v3.h[6]
                fmla v21.8h, v11.8h, v2.h[7]
                fmla v23.8h, v11.8h, v3.h[7]
                //  steps for output 9-12
                fmla v24.8h, v8.8h, v4.h[0]
                fmla v26.8h, v8.8h, v5.h[0]
                fmla v24.8h, v9.8h, v4.h[1]
                fmla v26.8h, v9.8h, v5.h[1]
                fmla v25.8h, v8.8h, v4.h[4]
                fmla v27.8h, v8.8h, v5.h[4]
                fmla v25.8h, v9.8h, v4.h[5]
                fmla v27.8h, v9.8h, v5.h[5]
                fmla v24.8h, v10.8h, v4.h[2]
                fmla v26.8h, v10.8h, v5.h[2]
                fmla v24.8h, v11.8h, v4.h[3]
                fmla v26.8h, v11.8h, v5.h[3]
                fmla v25.8h, v10.8h, v4.h[6]
                fmla v27.8h, v10.8h, v5.h[6]
                fmla v25.8h, v11.8h, v4.h[7]
                fmla v27.8h, v11.8h, v5.h[7]
                //  steps for output 13-16
                fmla v28.8h, v8.8h, v6.h[0]
                fmla v30.8h, v8.8h, v7.h[0]
                fmla v28.8h, v9.8h, v6.h[1]
                fmla v30.8h, v9.8h, v7.h[1]
                fmla v29.8h, v8.8h, v6.h[4]
                fmla v31.8h, v8.8h, v7.h[4]
                fmla v29.8h, v9.8h, v6.h[5]
                fmla v31.8h, v9.8h, v7.h[5]
                fmla v28.8h, v10.8h, v6.h[2]
                fmla v30.8h, v10.8h, v7.h[2]
                fmla v28.8h, v11.8h, v6.h[3]
                fmla v30.8h, v11.8h, v7.h[3]
                fmla v29.8h, v10.8h, v6.h[6]
                fmla v31.8h, v10.8h, v7.h[6]
                fmla v29.8h, v11.8h, v6.h[7]
                fmla v31.8h, v11.8h, v7.h[7]

                cbz x8, WriteRes
                // relu
                dup v8.4s, wzr
                fmax v16.8h, v16.8h, v8.8h
                fmax v17.8h, v17.8h, v8.8h
                fmax v18.8h, v18.8h, v8.8h
                fmax v19.8h, v19.8h, v8.8h
                fmax v20.8h, v20.8h, v8.8h
                fmax v21.8h, v21.8h, v8.8h
                fmax v22.8h, v22.8h, v8.8h
                fmax v23.8h, v23.8h, v8.8h
                fmax v24.8h, v24.8h, v8.8h
                fmax v25.8h, v25.8h, v8.8h
                fmax v26.8h, v26.8h, v8.8h
                fmax v27.8h, v27.8h, v8.8h
                fmax v28.8h, v28.8h, v8.8h
                fmax v29.8h, v29.8h, v8.8h
                fmax v30.8h, v30.8h, v8.8h
                fmax v31.8h, v31.8h, v8.8h

                cbz x9, WriteRes
                // relu 6
                movi v9.8h, #0x46, lsl #8
                fmin v16.8h, v16.8h, v9.8h
                fmin v17.8h, v17.8h, v9.8h
                fmin v18.8h, v18.8h, v9.8h
                fmin v19.8h, v19.8h, v9.8h
                fmin v20.8h, v20.8h, v9.8h
                fmin v21.8h, v21.8h, v9.8h
                fmin v22.8h, v22.8h, v9.8h
                fmin v23.8h, v23.8h, v9.8h
                fmin v24.8h, v24.8h, v9.8h
                fmin v25.8h, v25.8h, v9.8h
                fmin v26.8h, v26.8h, v9.8h
                fmin v27.8h, v27.8h, v9.8h
                fmin v28.8h, v28.8h, v9.8h
                fmin v29.8h, v29.8h, v9.8h
                fmin v30.8h, v30.8h, v9.8h
                fmin v31.8h, v31.8h, v9.8h

            WriteRes:

                // prefetching is not prefered while writing results in spite of cache missings
                // you could try prfm pstl2strm
                // there are almost no benefits observed though
                st1 {v16.8h}, [x11], x7
                st1 {v17.8h}, [x11], x7
                st1 {v18.8h}, [x11], x7
                st1 {v19.8h}, [x11], x7
                st1 {v20.8h}, [x11], x7
                st1 {v21.8h}, [x11], x7
                st1 {v22.8h}, [x11], x7
                st1 {v23.8h}, [x11], x7
                st1 {v24.8h}, [x11], x7
                st1 {v25.8h}, [x11], x7
                st1 {v26.8h}, [x11], x7
                st1 {v27.8h}, [x11], x7
                st1 {v28.8h}, [x11], x7
                st1 {v29.8h}, [x11], x7
                st1 {v30.8h}, [x11], x7
                st1 {v31.8h}, [x11]

            subs x10, x10, #1
            add x0, x0, #16
            bne LoopKw

        subs x6, x6, #8
        bgt LoopStart

    sub sp, sp, #64
    ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [sp], #64
    ret
#endif


